#!/usr/bin/env python
import os, sys
from datetime import datetime
import glob
import json
from lisn_utils import *

sys.path.append('../')
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dblisn.settings")

from apps.stations.models import Station
from apps.instruments.models import Instrument, InstrumentType
from apps.data.models import Data, DataType, DataDate, DataFile

BD = MySQL(host = "localhost",
           user = "lisn",
           passwd = "lisn",
           db = "lisn",
           #verbose=True
           )

req = LISNRequest()

TYPES = {'product':1, 'gps':2, 'magnetometer':3, 'ionosonde':4}
DATATYPES = {'rinex':['%j0.%y',4,11],
             'ion':['%Y%j',6,13],
             'other':['%y%m%d',5,11],}

def get_date(filename, datatype):
    '''
    '''
    try:
        if datatype=='rinex':
            return datetime.strptime(filename[DATATYPES['rinex'][1]:DATATYPES['rinex'][2]], DATATYPES['rinex'][0])
        elif datatype in ('binary','posit','scint', 'stats', 'tec', 'minute', 'second', 'minute_corrected'):
            return datetime.strptime(filename[DATATYPES['other'][1]:DATATYPES['other'][2]], DATATYPES['other'][0])
        elif datatype in ('dat','OX','PWR', 'SNR', 'EDP', 'ESR', 'SAO','ngi','RIQ'):
            return datetime.strptime(filename[DATATYPES['ion'][1]:DATATYPES['ion'][2]], DATATYPES['ion'][0])
        else:
            return False
    except:
        return False

def migrate_stations():
    rows = BD.select('stations', ['*'], '1')
    for tup in rows:
        values = {}
        values['name'] = tup[1]
        values['country'] = tup[2]
        values['latitude'] = tup[3]
        values['longitude'] = tup[4]
        values['altitude'] = tup[5]
        values['state'] = 1
        if not Station.objects.filter(name=tup[1]):
            model = Station(**values)
            model.save()
    model = Station(id=-1, name='all', country='all', latitude=0, longitude=0, altitude=0, state=0)
    model.save()    

def migrate_instruments():
    rows = BD.select('stations, instruments', ['*'], 'stations.idstation=instruments.idstation')
    for tup in rows:
        sta = Station.objects.get(name=tup[1])
        print sta
        values = {}
        values['station_id'] = sta.pk
        values['type_id'] = TYPES[tup[11]]
        values['lisn_code'] = tup[9]
        values['code'] = tup[10]
        values['network'] = tup[12]
        values['server'] = tup[13]
        values['state'] = tup[14]
        values['last_file'] = tup[15]
        values['last_file_date'] = tup[16]
        if tup[11]=='gps':
            info = json.dumps({'receiver':tup[17], 'antenna':tup[18], 'receiver_number':tup[19], 'antenna_number':tup[20], 'receiver_firmware':tup[21]})
        elif tup[11]=='magnetometer':
            info = json.dumps({'model':tup[17]})
        elif tup[11]=='ionosonde':
            info = json.dumps({'model':'VIPIR', 'TX Antenna':'Log Periodic'})
        values['info'] = info
        model = Instrument(**values)
        model.save()

def list_data():
    fields = ['path', 'filename', 'alias', 'dates', 'instruments.instrument', 'code']
    rows = BD.select('data,datatypes,instruments', fields, 'data.iddatatype=datatypes.iddatatype AND data.idinstrument=instruments.idinstrument')
    cnt = 0
    for tup in rows:
        cnt += 1
        xml = MyXML(tup['dates'])
        for lyear in xml.data:
            for lmonth in lyear:
                for lday in lmonth:
                    dt = GPSDateTime(int(lyear.text), int(lmonth.text), int(lday.text))
                    filename = os.path.join(dt.strftime(tup['path']), dt.strftime('*%s*' % tup['filename'].split(',')[0]))
                    if len(glob.glob(filename))==0:
                        print '%s,%s,%s,%s,%s' % (tup['instruments.instrument'], tup['code'], tup['alias'],dt.strftime('%y%m%d'),filename)
    print cnt

def migrate_datatypes():
    rows = BD.select('datatypes', ['*'], '1')
    for tup in rows:
        values = {}
        values['name'] = tup[2]
        values['alias'] = tup[3]
        values['state'] = 1
        model = DataType(**values)
        model.save()
        
def migrate_data_gps(network):
        dum = []
        sites = req.stations(instrument='magnetometer', network=network)
        path = '/data1/magnetometer/%s' % network        
        for code in os.listdir(path):
            if code not in [s['code'] for s in sites]:
                print 'code not found %s' % code
                continue
            path_code = os.path.join(path, code)
            print path_code
            ii = Instrument.objects.get(code=code, type_id=3)
            for year in os.listdir(path_code):
                path_year = os.path.join(path_code, year)
                if not os.path.isdir(path_year): continue
                for month in os.listdir(path_year):
                    path_month = os.path.join(path_year, month)
                    if not os.path.isdir(path_month): continue                    
                    for dtype in os.listdir(path_month):
                        if not os.path.isdir(os.path.join(path_month, dtype)) or dtype in ('bad_tec', 'tecmalo'): continue
                        dd = DataType.objects.get(alias=dtype)
                        if code+dtype in dum:
                            continue
                        else:
                            if Data.objects.filter(instrument_id=ii.pk, datatype_id=dd.pk):
                                continue
                            print ' %s' % dtype
                            dum.append(code+dtype)
                            values = {}
                            values['instrument_id'] = ii.pk
                            values['datatype_id'] = dd.pk
                            values['path'] = '/data1/magnetometer/%s/%s/%s/%s/%s' % (network,code,'%Y','%m',dtype)
                            values['state'] = 1
                            model = Data(**values)
                            model.save()

def migrate_data_ion():
        dum = []
        path = '/data1/ionosonde'        
        for code in os.listdir(path):
            path_code = os.path.join(path, code)
            print path_code
            ii = Instrument.objects.get(code=code, type_id=4)
            for year in os.listdir(path_code):
                path_year = os.path.join(path_code, year)
                if not os.path.isdir(path_year): continue
                for month in os.listdir(path_year):
                    path_month = os.path.join(path_year, month)
                    if not os.path.isdir(path_month): continue                    
                    for day in os.listdir(path_month):
                        path_day = os.path.join(path_month, day)
                        if not os.path.isdir(path_day): continue                         
                        for dtype in os.listdir(path_day):
                            if not os.path.isdir(os.path.join(path_day, dtype)) or dtype=='figures': continue                            
                            dd = DataType.objects.get(alias=dtype)
                            if code+dtype in dum:
                                continue
                            else:
                                print os.path.join(path_day, dtype)
                                if Data.objects.filter(instrument_id=ii.pk, datatype_id=dd.pk):
                                    continue
                                dum.append(code+dtype)
                                values = {}
                                values['instrument_id'] = ii.pk
                                values['datatype_id'] = dd.pk
                                values['path'] = '/data1/ionosonde/%s/%s/%s/%s/%s' % (code,'%Y','%m','%d',dtype)
                                values['state'] = 1
                                model = Data(**values)
                                model.save()

def migrate_files_gps(network):
        path = '/data1/gps/%s' % network
        sites = req.stations(instrument='gps', network=network)    
        for code in os.listdir(path):
            if code not in [s['code'] for s in sites]:
                print 'code not found %s' % code
                continue
            path_code = os.path.join(path, code)
            print path_code
            ii = Instrument.objects.get(code=code, type_id=2)
            for year in os.listdir(path_code):
                path_year = os.path.join(path_code, year)
                if not os.path.isdir(path_year): continue
                for month in os.listdir(path_year):
                    path_month = os.path.join(path_year, month)
                    if not os.path.isdir(path_month): continue                    
                    for dtype in os.listdir(path_month):
                        if not os.path.isdir(os.path.join(path_month, dtype)) or dtype in ('bad_tec', 'tecmalo'): continue
                        #if dtype!=alias: continue
                        dd = DataType.objects.get(alias=dtype)
                        path_dtype = os.path.join(path_month, dtype)
                        if not os.path.isdir(path_dtype): continue
                        data = Data.objects.get(instrument_id=ii.pk, datatype_id=dd.pk)
                        print ' %s' % data
                        for filename in os.listdir(path_dtype):
                            if filename.endswith('.png'):
                                continue
                            dt = get_date(filename, dtype)
                            #print dt
                            if not dt: continue
                            
                            if not DataDate.objects.filter(data_id=data.pk, date=dt):                                
                                values = {}
                                values['data_id'] = data.pk
                                values['date'] = dt
                                values['size'] = 0                                
                                values['state'] = 1
                                date = DataDate(**values)
                                date.save()                          
                            #break

def migrate_files_mag(network):
        path = '/data1/magnetometer/%s' % network        
        for code in os.listdir(path):
            path_code = os.path.join(path, code)
            print path_code
            ii = Instrument.objects.get(code=code, type_id=3)
            for year in os.listdir(path_code):
                path_year = os.path.join(path_code, year)
                if not os.path.isdir(path_year): continue
                for month in os.listdir(path_year):
                    path_month = os.path.join(path_year, month)
                    if not os.path.isdir(path_month): continue                    
                    for dtype in os.listdir(path_month):
                        if not os.path.isdir(os.path.join(path_month, dtype)) or dtype in ('bad_tec', 'tecmalo'): continue
                        dd = DataType.objects.get(alias=dtype)
                        path_dtype = os.path.join(path_month, dtype)
                        if not os.path.isdir(path_dtype): continue
                        data = Data.objects.get(instrument_id=ii.pk, datatype_id=dd.pk)
                        print ' %s, %s, %s' % (data, year, month)
                        for filename in os.listdir(path_dtype):
                            dt = get_date(filename, dtype)
                            if not dt: continue                            
                            if not DataDate.objects.filter(data_id=data.pk, date=dt):                                
                                values = {}
                                values['data_id'] = data.pk
                                values['date'] = dt
                                values['state'] = 1
                                date = DataDate(**values)
                                date.save()
                            else:
                                date = DataDate.objects.get(data_id=data.pk, date=dt)   
                            if not DataFile.objects.filter(datadate_id=date.pk, filename=filename):
                                values = {}
                                values['datadate_id'] = date.pk
                                values['filename'] = filename
                                values['filesize'] = os.path.getsize(os.path.join(path_dtype, filename))
                                values['state'] = 1
                                model = DataFile(**values)
                                model.save()



def update_inst_info():
    tups = Instrument.objects.all()
    for tup in tups:
        info = tup.info
        
        tup.save()

#update_inst_info()
migrate_files_gps('LISN')




    
